import re #regular expression library
import sys #writing to files
import urllib #url library
import string #string functions

def getUrlData(url):
    #TO-DO check connection
    urlSearch = url
    urlfile = urllib.urlopen(urlSearch)
    
    #urldata will contain the url data with out null chars
    urldata = urlfile.read().replace("\0","")
    urlfile.close()
    return urldata

def extractResultsLive(urldata):
    
    regexGetResultBlock = "\<div id=\"results\"\>.+?\</div\>"
    resultBlock = re.findall(regexGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock) #maxim wants to ask carpenter
    
    regexGetResultsUrls = "(?<=\<h3\>\<a href=\").+?(?=\")"
    resultsUrls = re.findall(regexGetResultsUrls, resultBlockAsString, re.DOTALL)
     
    return resultsUrls
#end extractResultsLive    
        
def extractResultsYahoo(urldata):  
    
    regexGetResultBlock = "\<h2\>WEB RESULTS\</h2\>.+?\</ol\>"
    resultBlock = re.findall(regexGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock) #maxim wants to ask carpenter
          
    regexGetResultsUrls = "(?<=\<div\>\<a class=yschttl href=\").+?(?=\")"
    resultsUrls = re.findall(regexGetResultsUrls, resultBlockAsString, re.DOTALL)
    
    return redirectedUrls(resultsUrls)
#end extractResultsYahoo

def getNextUrlYahoo(urldata):
    regexGetNextUrl = "(?<=id\=\"pg-next\" class\=\"pg\" href\=\").+?(?=\"\>Next)"
    nextUrl = re.findall(regexGetNextUrl, urldata, re.DOTALL)
    if len(nextUrl) != 0:
        return nextUrl[0]
    else:
        return ""
#end getNextUrlYahoo

def getNextUrlLive(urldata):   
    regexGetNextUrl = "(?<=class\=\"nP\" href\=\").+?(?=\"\>Next)"    
    nextUrl = re.findall(regexGetNextUrl, urldata, re.DOTALL)  
    if len(nextUrl) != 0: 
        fixNextUrl = string.replace(nextUrl[0], ";", "")
        fixNextUrl = string.replace(fixNextUrl, "amp", "")
        fixNextUrl = "http://search.live.com" + fixNextUrl 
        return fixNextUrl
    else:
        return ""
#end getNextUrlYahoo


def redirectedUrls(urls):
    count = 0
    for url in urls:
        if string.find(url, "http://rdre") != -1:
            #url = "http://tracking.voltagesearch.com/181-3640026-3-t2355721-2090-0-214909/2/&y=042FC1622395808643&i=482&c=48333&q=02%5ESSHPM%5BL7qvtz6&e=utf-8&r=7&d=wow~YHOO-en-us&n=K9P4G7LQKOO4F4EV&s=20&t=&m=4730D8B2&x=054A40B4CFDE235BFBF2BBE7373B67CAC0"

            urlfile = urllib.urlopen(url)
            urls[count] = urlfile.geturl()
        count = count + 1
    return urls
#end redirectedUrls

#def compareArrays(A, B):
#    matchesList = []
#    
#    for itemA in A:
#        for itemB in B:
#            if itemA == itemB:
#                matchesList.append(itemA)
#    matchesList            
#    return matchesList
#end compareArrays
                
def findMatches(list):
    matches = [] #list to save the matches
    i = 0
    while i<=len(list)-2 :
        if list[i] == list[i+1]: #If found match
            matches.append(list[i]) #save it into matches
            list.pop(i) #delete from the list twice(2 matches)
            list.pop(i)
        i+=1#keep going on the list
    return matches
            
#MAIN

targetMatches = 3
textQuery = "nike"
query = urllib.quote(textQuery) #replaces unsafe and special chars with %ascii 



liveData = getUrlData("http://search.live.com/results.aspx?q=" + query)
resultsLive = extractResultsLive(liveData)
print "Live:"
for url in resultsLive :
        print url      

yahooData = getUrlData("http://search.yahoo.com/search?p=" + query)
resultsYahoo = extractResultsYahoo(yahooData)
print "Yahoo:"
for url in resultsYahoo :
        print url
        
#matches = compareArrays(resultsYahoo, resultsLive)
#print "Matches:"
#for match in matches:
#    print match

results = []
results = resultsLive;
results.extend(resultsYahoo)
results.sort()

print "Matches(sorted array)"
print len(results)
matches = findMatches(results)
print len(results)
for each in matches:
    print each

#do {
#    extractY, extractL
#    results.extend(yahoo, live)
#    matches.extend(findmatches(resluts))
#    
#    } while matches<x


#ENDMAIN
    #TESTT   Aaother test 
    ##Open the output file for "w"riting
    #file = open("OPIM311Files\LiveSerachResults.html","w")
    #print >> file, resultBlock
    #file.close()b